﻿using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
using KeRui.Common;
using System.Text.RegularExpressions;
using System.Net;

public partial class Manage_Cms_CatchParseTest : System.Web.UI.Page
{

    private static RegexOptions regex = RegexOptions.IgnoreCase | RegexOptions.Multiline;

    public int CatchID = PageRequest.GetInt("CatchID", -1);
    public KeRui.Model.CmsServer ServerInfo = new KeRui.Model.CmsServer();

    protected void Page_Load(object sender, EventArgs e)
    {
        ManageHelper.CheckAdminLogin();

        //加载服务器信息
        KeRui.BLL.CmsServer bllServer = new KeRui.BLL.CmsServer();
        ServerInfo = bllServer.GetModelByCache(PageRequest.GetServerIP());

        //加载采集规则
        KeRui.BLL.CmsCatch bll = new KeRui.BLL.CmsCatch();
        KeRui.Model.CmsCatch model = bll.GetModel(CatchID);
        if (model == null)
        {
            Response.Write("无法加载采集规则。");
            Response.End();
        }
        //加载解析规则
        KeRui.BLL.CmsCatchParse bllParse = new KeRui.BLL.CmsCatchParse();
        KeRui.Model.CmsCatchParse parse = bllParse.GetModel(model.ParseID);

        if (parse == null)
        {
            Response.Write("无法加载解析器。");
            Response.End();
        }

        #region 采集处理
        MatchCollection matchesUrl = Regex.Matches(model.Url, "{date,(?<date>[^}]*?)}", regex);

        if (matchesUrl.Count > 0)
        {
            Match matchUrl = matchesUrl[0];
            GroupCollection ItemsUrl = matchUrl.Groups;
            model.Url = model.Url.Replace("{date," + ItemsUrl["date"].Value + "}", DateTime.Now.ToString(ItemsUrl["date"].Value));
        } 
        string htmlData = KeRui.BLL.CmsCatchHack.GetHtml(model.Url.Replace("{page}", model.StartPage.ToString()), parse.ListEncoding);

        if (htmlData == "")
        {
            Response.Write("下载失败，无法获得HTML。URL：" + model.Url);
            Response.End();
        }
        else
        {
            //Response.Write("<hr><p><span style='color:red'><b>↓抓取文章列表开始↓</b></span></p>\r\n\r\n\r\n\r\n");
            CatchContentList(model, 1);
            //Response.Write("<br><p><span style='color:red'><b>↑抓取文章列表结束↑</b></span></p><hr>\r\n\r\n\r\n\r\n");
        }
        #endregion

        Response.End();
    }

    /// <summary>
    /// 采集列表
    /// </summary>
    /// <param name="catchs">采集对象model</param>
    /// <param name="curPage">当前页面</param>
    void CatchContentList(KeRui.Model.CmsCatch catchs, int curPage)
    {

        if (catchs.StartPage <= 1)
            catchs.StartPage = 1;
        if (catchs.EndPage <= 1)
            catchs.EndPage = 1;
        if (curPage <= 1)
            curPage = 1;
        if (curPage < catchs.StartPage)
            curPage = catchs.StartPage;

        //退出
        if (curPage > catchs.EndPage)
            return;

        //加载解析规则
        KeRui.BLL.CmsCatchParse bllParse = new KeRui.BLL.CmsCatchParse();
        KeRui.Model.CmsCatchParse parse = bllParse.GetModel(catchs.ParseID);
        if (parse == null)
        {
            return;
        }

        string curUrl = catchs.Url.Replace("{page}", curPage.ToString());

        MatchCollection matchesUrl = Regex.Matches(curUrl, "{date,(?<date>[^}]*?)}", regex);

        if (matchesUrl.Count>0)
        {
            Match matchUrl = matchesUrl[0];
            GroupCollection ItemsUrl = matchUrl.Groups;
            curUrl = curUrl.Replace("{date," + ItemsUrl["date"].Value + "}", DateTime.Now.ToString(ItemsUrl["date"].Value));
        } 

        string htmlData = KeRui.BLL.CmsCatchHack.GetHtml(curUrl, parse.ListEncoding);

        if (htmlData != "")
        {
            if (parse != null)
            {
                //定位内容
                htmlData = Utils.GetContent(htmlData, parse.ListParseBegin, parse.ListParseEnd);

                #region CDATA 过滤
                MatchCollection matchesCDATA = Regex.Matches(htmlData, @"<(?<title1>[a-z]+)>[\s]*?<!\[CDATA\[(?<cdata>[^>]*?)\]\]>[\s]*?</(?<title2>[a-z]+)>", regex);
                foreach (Match matchCDATA in matchesCDATA)
                {
                    GroupCollection ItemsCDATA = matchCDATA.Groups;
                    htmlData = htmlData.Replace(@"<" + ItemsCDATA["title1"].Value + @"><![CDATA[" + ItemsCDATA["cdata"].Value + @"]]></" + ItemsCDATA["title2"].Value + @">", @"<" + ItemsCDATA["title1"].Value + @">" + ItemsCDATA["cdata"].Value + @"</" + ItemsCDATA["title2"].Value + @">");
                }
                #endregion

                //XHTML格式化
                htmlData = Utils.FilterXss(htmlData, parse.ListParseTags, parse.ListParseAttrs);

                //正则匹配列表
                MatchCollection matches = Regex.Matches(htmlData, parse.ListParseRule, regex);

                if (matches.Count == 0)
                {
                    Response.Write("没有分析到文章列表。");
                    Response.End();
                    return;
                }

                int iii = 1;
                foreach (Match match in matches)
                {
                    GroupCollection Items = match.Groups;

                    //获取URL
                    string Url = parse.ContentUrl;
                    for (int i = 0; i < Items.Count-1; i++)
                    {
                        Url = Url.Replace("{" + i + "}", Items[i + 1].Value.Trim());
                    }

                    //获得标题（只用于显示）
                    string Title = Items["title"].Value.Trim();
                    //移除HTML标签
                    Title = Utils.RemoveHtml(Title);
                    //获得图片
                    string Pic = Items["pic"].Value.Trim();
                    //修正图片
                    if (Pic != "")
                        Pic = KeRui.BLL.CmsCatchHack.HackUrl(Url, Pic);

                    Response.Write("<p id=" + iii + ">"+iii+".   [<a href='CatchParseTest.aspx?CatchID=" + catchs.CatchID + "&num=" + iii + "#" + iii + "'>详</a>] ");
                    Response.Write("<span style='color:red'><b>" + Title + "</b></span> ");
                    Response.Write("<a href='" + Url + "' target='_blank'>网址</a> ");
                    if(Pic!="")
                        Response.Write("<a href='" + Pic + "' target='_blank'>图片</a> ");
                    Response.Write("</p>\r\n\r\n\r\n\r\n");

                    //采集内容
                    if (iii == PageRequest.GetInt("num", 0))
                        CatchContentText(catchs, parse, Url,Title,Pic,-1);
                    iii++;
                }
            }
        }
        else
        {
            Response.Write("下载失败，无法获得HTML。URL：" + curUrl);
            Response.End();
        }

        //采集下一页
        //if (curPage + 1 <= catchs.EndPage)
        //    CatchContentList(catchs, curPage + 1);
    }



    /// <summary>
    /// 采集内容
    /// </summary>
    /// <param name="catchs">采集对象model</param>
    /// <param name="parse">内容解析model</param>
    /// <param name="url">内容地址</param>
    void CatchContentText(KeRui.Model.CmsCatch catchs, KeRui.Model.CmsCatchParse parse, string url, string title, string pic,int contentid)
    {
        KeRui.BLL.CmsCatchCache bllCache = new KeRui.BLL.CmsCatchCache();
        //KeRui.Model.CmsCatchCache modelCache = bllCache.GetModel(url);
        //已经采集过了。
        //if (modelCache != null)
        //    return;

        //源数据
        string htmlData = KeRui.BLL.CmsCatchHack.GetHtml(url, parse.ContentEncoding);

        //原始内容
        string htmlDataFull = htmlData;

        //新内容ID，给采集用
        int ContentID = -1;

        if (htmlData != "")
        {
            //内容解析B
            KeRui.BLL.CmsCatchParseRegex bllRegex = new KeRui.BLL.CmsCatchParseRegex();

            //定位内容
            htmlData = Utils.GetContent(htmlData, parse.ContentParseBegin, parse.ContentParseEnd);

            //XHTML格式化
            htmlData = Utils.FilterXss(htmlData, parse.ContentParseTags, parse.ContentParseAttrs);

            //正则匹配内容
            MatchCollection matches = Regex.Matches(htmlData, parse.ContentParseRule, regex);

            //捕获自定义表单或获得内容（正则）
            MatchCollection matches2 = Regex.Matches(parse.ContentParseRule, @"\(\?\<(?<fieldname>[0-9a-zA-Z]*?)\>(?<pattern>[\s\S]*?)\)", regex);

            foreach (Match match in matches)
            {
                GroupCollection Items = match.Groups;

                //内容保存
                string PostStr = "_act=catch";
                PostStr += "&_nodeid=" + Utils.UrlEncode(catchs.NodeID.ToString(), "utf-8");

                //处理catchs里的设置
                int ContentState = 1;//0:草稿 1:待审核 2:审核成功 3:审核失败
                int PublicState = 0;//发布状态 0:待发布 1:发布中 2:自动发布

                string Title = "";
                string PicUrl = "";
                string Context = "";//对应Content

                #region 组织节点
                foreach (Match match2 in matches2)
                {
                    GroupCollection Items2 = match2.Groups;
                    string FieldName = Items2["fieldname"].Value.Trim().ToLower();
                    string Pattern = Items2["pattern"].Value.Trim();
                    string Content = Items[FieldName].Value.Trim();

                    //捕获指定字段Title内容
                    if (FieldName == "title")
                        Title = Content;
                    //捕获指定字段PicUrl内容
                    if (FieldName == "picurl")
                        PicUrl = Content;
                    //捕获指定字段Content内容
                    if (FieldName == "content")
                        Context = Content;

                    //【所有字段】加载内容过滤
                    #region CmsCatchParseRegex
                    foreach (KeRui.Model.CmsCatchParseRegex model in bllRegex.GetList(parse.ParseID))
                    {
                        if (FieldName == model.FieldName.ToLower())
                        {
                            Content = Regex.Replace(Content, model.Pattern, model.Replacement, regex);
                        }
                    }
                    #endregion

                    //【content】检查特殊Hack
                    #region
                    if (FieldName == "content")
                    {
                        if (KeRui.BLL.CmsCatchHack.CheckHack(Content))
                        {
                            //163flv
                            if (KeRui.BLL.CmsCatchHack.GetHackName(Content) == "163flv")
                            {
                                //真实数据
                                Content = KeRui.BLL.CmsCatchHack.GetHackContent(Content);

                                PicUrl = KeRui.BLL.CmsCatchHack.Hack163FlvPic(Content);

                                /*
                                //获得LinkUrl
                                string LinkUrl = KeRui.BLL.CmsCatchHack.Hack163FlvPic(Content);
                                //下载
                                LinkUrl = KeRui.BLL.CmsCatchHack.SavePic(url, LinkUrl, ServerInfo);
                                //输出
                                PostStr += "&linkurl=" + Utils.UrlEncode(LinkUrl, "utf-8");
                                Response.Write("<span style='color:blue'><b>linkurl</b></span>：" + LinkUrl + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                                */
                                //Hack数据
                                Content = KeRui.BLL.CmsCatchHack.Hack163Flv(Content);
                            }
                        }
                    }
                    #endregion


                    //【Context】1、抓取关键词黑名单Blacklist。逗号隔开，不入库。
                    #region Blacklist
                    if (Context != "")
                    {
                        string[] Blacklist = Utils.SplitString(catchs.Blacklist, ",");
                        for (int i = 0; i < Blacklist.Length; i++)
                        {
                            if (Blacklist[i] != "")
                            {
                                if (Context.ToLower().Contains(Blacklist[i].ToLower()))
                                    return;
                            }
                        }
                    }
                    #endregion

                    //【Context】1、抓取关键词白名单Whilelist。逗号隔开，自动审核+发布
                    #region Whilelist
                    if (Context != "")
                    {
                        if (catchs.Whilelist.ToLower().Trim() == "all")
                        {
                            ContentState = 2;
                            PublicState = 1;
                        }
                        else
                        {
                            string[] Whilelist = Utils.SplitString(catchs.Whilelist, ",");
                            for (int i = 0; i < Whilelist.Length; i++)
                            {
                                if (Whilelist[i] != "")
                                {
                                    if (Context.ToLower().Contains(Whilelist[i].ToLower()))
                                    {
                                        ContentState = 2;
                                        PublicState = 1;
                                    }
                                }
                            }
                        }
                    }
                    #endregion

                    //图片保存
                    #region SavePic
                    if (catchs.SavePic == 1)
                    {
                        MatchCollection matchesImg = Regex.Matches(Content, @"<img([\s\S]*?)src=""(?<img>[\s\S]*?)""([\s\S]*?)>", regex);
                        #region 图片存储
                        foreach (Match matchImg in matchesImg)
                        {
                            GroupCollection ItemsImg = matchImg.Groups;

                            string ImgUrl = KeRui.BLL.CmsCatchHack.SavePic(url, ItemsImg["img"].Value.Trim(), ServerInfo);

                            //Response.Write("<hr>" + ImgUrl + "<hr>");

                            //<img([\s\S]*?)src无效，得使用<img src，但是中间有可能有内容，所以先用Replace，不用Regex.Replace
                            //Content = Regex.Replace(Content, @"<img([\s\S]*?)src=""" + ItemsImg["img"].Value.Trim() + @"""([\s\S]*?)>", @"<img src=""" + ImgUrl + @""" />", regex);
                            Content = Content.Replace(ItemsImg["img"].Value.Trim(), ImgUrl);

                        }
                        #endregion

                    }
                    #endregion

                    PostStr += "&" + FieldName + "=" + Utils.UrlEncode(Content, "utf-8");

                    Response.Write("<span style='color:blue'><b>" + FieldName + "</b></span>：" + Content + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");

                }
                #endregion

                //内容补充
                #region
                //PicUrl
                if (PicUrl == "")
                    PicUrl = pic;
                if (PicUrl != "")
                {
                    //下载
                    PicUrl = KeRui.BLL.CmsCatchHack.SavePic(url, PicUrl, ServerInfo);
                    //输出
                    PostStr += "&picurl=" + Utils.UrlEncode(PicUrl, "utf-8");
                    Response.Write("<span style='color:blue'><b>picurl</b></span>：" + PicUrl + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                }
                //ContentState
                PostStr += "&contentstate=" + ContentState + "";
                Response.Write("<span style='color:blue'><b>contentstate</b></span>：" + ContentState + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                //PublicState
                PostStr += "&publicstate=" + PublicState + "";
                Response.Write("<span style='color:blue'><b>publicstate</b></span>：" + PublicState + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                //ContentType
                PostStr += "&contenttype=" + catchs.ContentType + "";
                Response.Write("<span style='color:blue'><b>contenttype</b></span>：" + catchs.ContentType + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                //NodeID
                PostStr += "&nodeid=" + catchs.NodeID + "";
                Response.Write("<span style='color:blue'><b>nodeid</b></span>：" + catchs.NodeID + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                //ContentID
                PostStr += "&contentid=" + contentid + "";
                Response.Write("<span style='color:blue'><b>contentid</b></span>：" + contentid + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                //ContentPlus补充

                if (!string.IsNullOrEmpty(catchs.ContentPlus))
                {
                    string[] arrContentPlus = Utils.SplitString(catchs.ContentPlus, "&");
                    for (int arri = 0; arri < arrContentPlus.Length; arri++)
                    {
                        string[] arrContentValue = Utils.SplitString(arrContentPlus[arri], "=");
                        if (arrContentValue.Length == 2)
                        {
                            PostStr += "&" + arrContentValue[0] + "=" + Utils.UrlEncode(arrContentValue[1], "utf-8") + "";
                            Response.Write("<span style='color:blue'><b>" + arrContentValue[0] + "</b></span>：" + arrContentValue[1] + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                        }
                    }
                }
                //通信密钥
                string ServerKey = ServerInfo.ServerKey;
                PostStr += "&_m=" + Utils.MD5(ServerKey + ServerInfo.ServerIP);//ServerInfo全局变量
                Response.Write("<span style='color:blue'><b>_m</b></span>：" + Utils.MD5(ServerKey + ServerInfo.ServerIP) + "\r\n\r\n\r\n\r\n<br>\r\n\r\n\r\n\r\n");
                #endregion

                //保存数据，现在不保存
                #region
                /*if (modelCache != null)
                {
                    ContentID = modelCache.ContentID;
                }
                else
                {
                    KeRui.BLL.CmsContentInterface bllContent = new KeRui.BLL.CmsContentInterface();
                    //启动了才保存到数据
                    if (catchs.IsUsing == 1)
                    {
                        ContentID = bllContent.CatchContent(catchs.NodeID, PostStr);
                        //缓存记录
                        modelCache = new KeRui.Model.CmsCatchCache();
                        modelCache.Url = url;
                        modelCache.Title = Title;
                        modelCache.CatchID = catchs.CatchID;
                        modelCache.CatchName = catchs.CatchName;
                        modelCache.NodeID = catchs.NodeID;
                        modelCache.NodeName = catchs.NodeName;
                        modelCache.ContentID = ContentID;
                        bllCache.Add(modelCache);
                    }
                }*/
                #endregion

                //Response.Write("\r\n\r\n\r\n\r\n<hr color=red>" + PostStr + "\r\n\r\n\r\n\r\n\r\n\r\n\r\n");

            }
            Response.Write("\r\n\r\n\r\n\r\n<hr color=red>\r\n\r\n\r\n\r\n");
        }


        //检查分页
        #region
        if (parse.ContentPageParseRule != "")
        {
            string NextUrl = "";
            MatchCollection matchesPage = Regex.Matches(htmlDataFull, parse.ContentPageParseRule, regex);
            foreach (Match matchPage in matchesPage)
            {
                GroupCollection ItemsPage = matchPage.Groups;
                NextUrl = ItemsPage["url"].Value.Trim();
                //Response.Write("下一页：" + NextUrl+"<hr>");
            }
            if (NextUrl != "")
            {
                NextUrl = KeRui.BLL.CmsCatchHack.HackUrl(url, NextUrl);
                string htmlData2 = KeRui.BLL.CmsCatchHack.GetHtml(NextUrl, parse.ContentEncoding);
                if (htmlData2 != "")
                {
                    Response.Write("分页：" + NextUrl + "<hr color=red>");
                    CatchContentText(catchs, parse, NextUrl, title, pic, ContentID);
                }
            }
        }
        #endregion
    }

}

///HtmlString 暂不用
/*
public class HtmlString
{
    public HtmlString()
    { }
    private int _state;
    private string _html;
    public int State
    {
        set { _state = value; }
        get { return _state; }
    }
    public int Html
    {
        set { _html = value; }
        get { return _html; }
    }
}
*/